Advanced Lane Finding Project

The goals / steps of this project are the following:

• Compute the camera calibration matrix and distortion coefficients given a set of chessboard images.
• Apply a distortion correction to raw images.
• Use color transforms, gradients, etc., to create a thresholded binary image.
• Apply a perspective transform to rectify binary image ("birds-eye view").
• Detect lane pixels and fit to find the lane boundary.
• Determine the curvature of the lane and vehicle position with respect to center.
• Warp the detected lane boundaries back onto the original image.
• Output visual display of the lane boundaries and numerical estimation of lane curvature and vehicle position.

Camera Calibration

Camera Calibration using Chessboard images

import numpy as np
import cv2
import glob
import matplotlib.pyplot as plt
import pickle
import json

grid_x = 9
grid_y = 6
# prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
objp = np.zeros((grid_y*grid_x,3), np.float32)
objp[:,:2] = np.mgrid[0:grid_x, 0:grid_y].T.reshape(-1,2)

# Arrays to store object points and image points from all the images.
objpoints = [] # 3d points in real world space
imgpoints = [] # 2d points in image plane.
imgs = []

# Make a list of calibration images
img_fnames = glob.glob('camera_cal/calibration*.jpg')

# Step through the list and search for chessboard corners
for idx, fname in enumerate(img_fnames):
    img = cv2.imread(fname)
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Find the chessboard corners
    ret, corners = cv2.findChessboardCorners(gray, (grid_x, grid_y), None)

    # If found, add object points, image points
    if ret == True:
        objpoints.append(objp)
        imgpoints.append(corners)

        # Draw and display the corners
        cv2.drawChessboardCorners(img, (8,6), corners, ret)
        imgs.append(img)
        
h, w = imgs[0].shape[:2]
img_size = (w, h)
ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, img_size,None,None)

# Save the camera calibration result for later use (we won't worry about rvecs / tvecs)
np.savez('camera_cal.npz', mtx=mtx, dist=dist)

Visualization of Undistortion

%matplotlib inline

f, subplots = plt.subplots(2, 3, figsize=(20,10))

for idx, img in enumerate(imgs[:3]):
    dst = cv2.undistort(img, mtx, dist, None, mtx)
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    dst = cv2.cvtColor(dst, cv2.COLOR_BGR2RGB)
    subplots[0][idx].imshow(img)
    subplots[0][idx].set_title('Original Image', fontsize=10)
    subplots[1][idx].imshow(dst)
    subplots[1][idx].set_title('Undistorted Image', fontsize=10)

Thresholding

Pipeline on Test Images

def show_imgs(imgs1, imgs2, title1, title2):
    h = len(imgs1)
    f, subplots = plt.subplots(h, 2, figsize=(20, 5*h))

    for idx, img_pair in enumerate(zip(imgs1, imgs2)):
        img1, img2 = img_pair
        if len(img1.shape) == 3:
            img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)
            subplots[idx][0].imshow(img1)
        else:
            subplots[idx][0].imshow(img1, cmap='gray')
        subplots[idx][0].set_title(title1, fontsize=20)
        if len(img2.shape) == 3:
            img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
            subplots[idx][1].imshow(img2)
        else:
            subplots[idx][1].imshow(img2, cmap='gray')
        subplots[idx][1].set_title(title2, fontsize=20)

import os

img_fnames = ['test_images/straight_lines1.jpg', 'test_images/straight_lines2.jpg',
             'test_images/test1.jpg', 'test_images/test2.jpg', 'test_images/test3.jpg',
             'test_images/test4.jpg', 'test_images/test4.jpg', 'test_images/test6.jpg',]
original_imgs = []
undistorted_imgs = []

for idx, fname in enumerate(img_fnames):
    img = cv2.imread(fname)
    original_imgs.append(img)
    
    dst = cv2.undistort(img, mtx, dist, None, mtx)
    undistorted_imgs.append(dst)
    out = os.path.join('./output_images/undistortion', os.path.basename(fname))
    cv2.imwrite(out, dst)

show_imgs(original_imgs[:3], undistorted_imgs[:3], "original", "undistorted")

Gradients Thresholding

X,Y Gradient

# Define a function that takes an image, gradient orientation,
# and threshold min / max values.
def abs_sobel_thresh(img, orient='x', sobel_kernel=3, thresh=(0, 255)):
    # Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Apply x or y gradient with the OpenCV Sobel() function
    # and take the absolute value
    if orient == 'x':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel))
    if orient == 'y':
        abs_sobel = np.absolute(cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel))
    # Rescale back to 8 bit integer
    scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
    # Create a copy and apply the threshold
    binary_output = np.zeros_like(scaled_sobel)
    # Here I'm using inclusive (>=, <=) thresholds, but exclusive is ok too
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1

    # Return the result
    return binary_output
    

# Run the function
image = undistorted_imgs[0]
grad_binary = abs_sobel_thresh(image, orient='x', thresh=(20,100))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
ax1.set_title('Original Image', fontsize=20)
ax2.imshow(grad_binary, cmap='gray')
ax2.set_title('Thresholded Gradient', fontsize=20)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Magnitude of the Gradient

def mag_thresh(img, sobel_kernel=3, mag_thresh=(0, 255)):
    
    # Apply the following steps to img
    # 1) Convert to grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # 2) Take the gradient in x and y separately
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # 3) Calculate the magnitude 
    abs_sobelxy = np.sqrt(np.square(sobelx) + np.square(sobely))
    # 4) Scale to 8-bit (0 - 255) and convert to type = np.uint8
    scaled_sobelxy = np.uint8(255*abs_sobelxy/np.max(abs_sobelxy))
    # 5) Create a binary mask where mag thresholds are met
    binary_output = np.zeros_like(scaled_sobelxy)
    binary_output[(scaled_sobelxy >= mag_thresh[0]) & (scaled_sobelxy <= mag_thresh[1])] = 1
    # 6) Return this mask as your binary_output image
    return binary_output

# Run the function
mag_binary = mag_thresh(image, sobel_kernel=3, mag_thresh=(30, 100))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(mag_binary, cmap='gray')
ax2.set_title('Thresholded Magnitude', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Direction of the Gradient

# Define a function to threshold an image for a given range and Sobel kernel
def dir_threshold(img, sobel_kernel=3, thresh=(0, np.pi/2)):
    # Grayscale
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Calculate the x and y gradients
    sobelx = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    sobely = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    # Take the absolute value of the gradient direction, 
    # apply a threshold, and create a binary image result
    absgraddir = np.arctan2(np.absolute(sobely), np.absolute(sobelx))
    binary_output =  np.zeros_like(absgraddir)
    binary_output[(absgraddir >= thresh[0]) & (absgraddir <= thresh[1])] = 1

    # Return the binary image
    return binary_output

# Run the function
dir_binary = dir_threshold(image, sobel_kernel=15, thresh=(0.7, 1.1))
# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(dir_binary, cmap='gray')
ax2.set_title('Thresholded Grad. Dir.', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

Combining Gradient-Thresholding

def gradient_threshold(image):
    ksize = 3
    # Apply each of the thresholding functions
    gradx = abs_sobel_thresh(image, orient='x', sobel_kernel=ksize, thresh=(20, 100))
    grady = abs_sobel_thresh(image, orient='y', sobel_kernel=ksize, thresh=(20, 100))
    mag_binary = mag_thresh(image, sobel_kernel=ksize, mag_thresh=(30, 100))
    dir_binary = dir_threshold(image, sobel_kernel=ksize, thresh=(0.7, 1.3))

    combined = np.zeros_like(dir_binary)
    combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))] = 1
    #combined[((gradx == 1) & (grady == 1))] = 1
    
    return combined

gradient_imgs = []
for idx, fname in enumerate(img_fnames):
    img = undistorted_imgs[idx]
    dst = gradient_threshold(img)
    gradient_imgs.append(dst)
    
    out = os.path.join('./output_images/gradient', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(undistorted_imgs[:3], gradient_imgs[:3], "undistorted", "combined gradient thresholding")

HLS and Color Thresholds

# Define a function that thresholds the S-channel of HLS
def hls_select(img, thresh=(0, 255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    s_channel = hls[:,:,2]
    binary_output = np.zeros_like(s_channel)
    binary_output[(s_channel > thresh[0]) & (s_channel <= thresh[1])] = 1
    return binary_output

hls_binary = hls_select(image, thresh=(155, 255))

# Plot the result
f, (ax1, ax2) = plt.subplots(1, 2, figsize=(24, 9))
f.tight_layout()
ax1.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
ax1.set_title('Original Image', fontsize=50)
ax2.imshow(hls_binary, cmap='gray')
ax2.set_title('Thresholded S', fontsize=50)
plt.subplots_adjust(left=0., right=1, top=0.9, bottom=0.)

hls_imgs = []
for idx, fname in enumerate(img_fnames):
    img = undistorted_imgs[idx]
    dst = hls_select(img, thresh=(150, 255))
    hls_imgs.append(dst)
    
    out = os.path.join('./output_images/hls', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(undistorted_imgs[2:], hls_imgs[2:], "undistored", "hls thresholding")

Combining Color and Gradient

def combined_threshold(image):
    ksize = 3
    # Apply each of the thresholding functions
    gradx = abs_sobel_thresh(image, orient='x', sobel_kernel=ksize, thresh=(20, 100))
    grady = abs_sobel_thresh(image, orient='y', sobel_kernel=ksize, thresh=(20, 100))
    mag_binary = mag_thresh(image, sobel_kernel=ksize, mag_thresh=(30, 100))
    dir_binary = dir_threshold(image, sobel_kernel=15, thresh=(0.7, 1.3))
    
    hls_binary = hls_select(image, thresh=(130, 255))

    combined = np.zeros_like(dir_binary)
    combined[((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1)) | (hls_binary == 1)] = 1

    return combined

combined_imgs = []
for idx, fname in enumerate(img_fnames):
    img = undistorted_imgs[idx]
    dst = combined_threshold(img)
    combined_imgs.append(dst)
    
    out = os.path.join('./output_images/combined', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(undistorted_imgs[2:], combined_imgs[2:], "undistored", "combined final")

Transform

W, H = 1500, 1280
trim_w, trim_h = 1500, 1000
src = np.array([[594, 450], [684, 450], [1056, 690], [250,690]], np.float32)
dst = np.array([[250, 0], [1056, 0], [1056, H], [250, H]], np.float32)

M = cv2.getPerspectiveTransform(src, dst)
Minv = cv2.getPerspectiveTransform(dst, src)

# Save the result for later use
np.savez('warp.npz', M=M, Minv=Minv)

def unwarp_trim(img):
    warped = cv2.warpPerspective(img, M, (W, H), flags=cv2.INTER_LINEAR)
    #delete the next two lines
    return warped[H-trim_h:]

def recover(img):
    h, w = img.shape[:2]
    new = np.zeros((H, W, 3), np.uint8)
    new[H-h:] = img
    
    return new

warped_imgs = []
for idx, fname in enumerate(img_fnames):
    img = undistorted_imgs[idx]
    dst = unwarp_trim(img)
    warped_imgs.append(dst)
    
    out = os.path.join('./output_images/warped', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(undistorted_imgs[2:], warped_imgs[2:], "undistored", "warped")

def noise_filter(image):
    kernel = np.ones((5,5),np.uint8)
    image = cv2.erode(image,kernel,iterations = 1)
    image = cv2.dilate(image,kernel,iterations = 1)
    return image

binary_warped_imgs = []
for idx, fname in enumerate(img_fnames):
    img = undistorted_imgs[idx]
    img = combined_threshold(img)
    dst = unwarp_trim(img)
    
    dst = noise_filter(dst)

    binary_warped_imgs.append(dst)
    
    out = os.path.join('./output_images/binary_warped', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(combined_imgs[2:], binary_warped_imgs[2:], "binary", "binary_warped")

Implement Sliding Windows and Fit a Polynomial

import numpy as np
import cv2
import matplotlib.pyplot as plt

# Define a class to receive the characteristics of each line detection
class Line():
    def __init__(self, name):

        self.name = name
        # numbers of interations
        self.n = 5
        # was the line detected in the last iteration?
        self.detected = False
        # x values of the last n fits of the line
        self.recent_xfitted = []
        # the last n fits of the line
        self.recent_fits = []
        #average x values of the fitted line over the last n iterations
        self.bestx = None
        #polynomial coefficients averaged over the last n iterations
        self.best_fit = None
        #polynomial coefficients for the most recent fit
        self.current_fit = None
        #radius of curvature of the line in some units
        self.radius_of_curvature = None
        #distance in meters of vehicle center from the line
        self.line_base_pos = None
        #difference in fit coefficients between last and new fits
        self.diffs = np.array([0,0,0], dtype='float')
        #x values for detected line pixels
        self.allx = None
        #y values for detected line pixels
        self.ally = None
        # threshold for difference
        self.threshold_fit = 700
        self.threshold_fitx = 200
        self.filter = ''

    def check_outlier(self, fit, fitx):

        self.diffs = np.abs(self.current_fit - fit)
        self.diffx = np.mean(np.abs(self.bestx - fitx))
        if self.name == self.filter:
            print(self.name)
            print(self.diffs, np.linalg.norm(self.diffs))
            print(self.diffx)

        if np.linalg.norm(self.diffs) > self.threshold_fit:
            return False
        if self.diffx > self.threshold_fitx:
            return False

        return True

    def set(self, fit, fitx):
        self.detected = True
        self.current_fit = fit
        self.recent_fits.append(fit)
        if len(self.recent_fits) > self.n:
            self.recent_fits.pop(0)
        self.best_fit = np.mean(np.vstack(self.recent_fits), axis=0)

        self.recent_xfitted.append(fitx)
        if len(self.recent_xfitted) > self.n:
            self.recent_xfitted.pop(0)

        self.bestx = np.mean(np.array(self.recent_xfitted), axis=0)

    def update(self, fit, ploty):
        y_eval = np.max(ploty)
        self.radius_of_curvature = ((1 + (2*fit[0]*y_eval + fit[1])**2)**1.5) / np.absolute(2*fit[0])
        fitx = fit[0]*ploty**2 + fit[1]*ploty + fit[2]
        if self.current_fit is not None:

            if self.check_outlier(fit, fitx):
                self.set(fit, fitx)
            else:
                if self.name == self.filter:
                    print("skip this frame")
                self.detected = False
                if len(self.recent_fits) > self.n:
                    self.recent_fits.pop(0)
        else:
            self.current_fit = fit
            self.set(fit, fitx)

        return self.bestx

class Lane():
    def __init__(self):
        self.left_fit = None
        self.right_fit = None
        self.MARGIN = 50
        self.size_threshold = 200

        self.left_line = Line("LEFT")
        self.right_line = Line("RIGHT")

    def find_lane_lines_first(self, binary_warped):
        # Assuming you have created a warped binary image called "binary_warped"
        # Take a histogram of the bottom half of the image
        histogram = np.sum(binary_warped[binary_warped.shape[0]//2:,:], axis=0)
        # Create an output image to draw on and  visualize the result
        out_img = np.dstack((binary_warped, binary_warped, binary_warped))*255
        # Find the peak of the left and right halves of the histogram
        # These will be the starting point for the left and right lines
        midpoint = np.int(histogram.shape[0]/2)
        leftx_base = np.argmax(histogram[:midpoint])
        rightx_base = np.argmax(histogram[midpoint:]) + midpoint

        # Choose the number of sliding windows
        nwindows = 9
        # Set height of windows
        window_height = np.int(binary_warped.shape[0]/nwindows)
        # Identify the x and y positions of all nonzero pixels in the image
        nonzero = binary_warped.nonzero()
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Current positions to be updated for each window
        leftx_current = leftx_base
        rightx_current = rightx_base
        # Set the width of the windows +/- margin
        margin = self.MARGIN
        # Set minimum number of pixels found to recenter window
        minpix = 50
        # Create empty lists to receive left and right lane pixel indices
        left_lane_inds = []
        right_lane_inds = []

        # Step through the windows one by one
        for window in range(nwindows):
            # Identify window boundaries in x and y (and right and left)
            win_y_low = binary_warped.shape[0] - (window+1)*window_height
            win_y_high = binary_warped.shape[0] - window*window_height
            win_xleft_low = leftx_current - margin
            win_xleft_high = leftx_current + margin
            win_xright_low = rightx_current - margin
            win_xright_high = rightx_current + margin
            # Draw the windows on the visualization image
            cv2.rectangle(out_img,(win_xleft_low,win_y_low),(win_xleft_high,win_y_high),(0,255,0), 2)
            cv2.rectangle(out_img,(win_xright_low,win_y_low),(win_xright_high,win_y_high),(0,255,0), 2)
            # Identify the nonzero pixels in x and y within the window
            good_left_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xleft_low) & (nonzerox < win_xleft_high)).nonzero()[0]
            good_right_inds = ((nonzeroy >= win_y_low) & (nonzeroy < win_y_high) & (nonzerox >= win_xright_low) & (nonzerox < win_xright_high)).nonzero()[0]
            # Append these indices to the lists
            left_lane_inds.append(good_left_inds)
            right_lane_inds.append(good_right_inds)
            # If you found > minpix pixels, recenter next window on their mean position
            if len(good_left_inds) > minpix:
                leftx_current = np.int(np.mean(nonzerox[good_left_inds]))
            if len(good_right_inds) > minpix:
                rightx_current = np.int(np.mean(nonzerox[good_right_inds]))

        # Concatenate the arrays of indices
        left_lane_inds = np.concatenate(left_lane_inds)
        right_lane_inds = np.concatenate(right_lane_inds)

        # Extract left and right line pixel positions
        leftx = nonzerox[left_lane_inds]
        lefty = nonzeroy[left_lane_inds]
        rightx = nonzerox[right_lane_inds]
        righty = nonzeroy[right_lane_inds]

        # Fit a second order polynomial to each
        left_fit = None
        right_fit = None
        if len(leftx) > self.size_threshold and len(lefty) > self.size_threshold:
            left_fit = np.polyfit(lefty, leftx, 2)
        if len(rightx) > self.size_threshold and len(righty) > self.size_threshold:
            right_fit = np.polyfit(righty, rightx, 2)

        return left_fit, right_fit

    def find_lane_lines_after(self, binary_warped, left_fit, right_fit):
        # Assume you now have a new warped binary image
        # from the next frame of video (also called "binary_warped")
        # It's now much easier to find line pixels!
        nonzero = binary_warped.nonzero()
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        margin = self.MARGIN
        left_lane_inds = ((nonzerox > (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] - margin)) & (nonzerox < (left_fit[0]*(nonzeroy**2) + left_fit[1]*nonzeroy + left_fit[2] + margin)))
        right_lane_inds = ((nonzerox > (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] - margin)) & (nonzerox < (right_fit[0]*(nonzeroy**2) + right_fit[1]*nonzeroy + right_fit[2] + margin)))

        # Again, extract left and right line pixel positions
        leftx = nonzerox[left_lane_inds]
        lefty = nonzeroy[left_lane_inds]
        rightx = nonzerox[right_lane_inds]
        righty = nonzeroy[right_lane_inds]
        # Fit a second order polynomial to each
        left_fit = None
        right_fit = None
        if len(leftx) > self.size_threshold and len(lefty) > self.size_threshold:
            left_fit = np.polyfit(lefty, leftx, 2)
        if len(rightx) > self.size_threshold and len(righty) > self.size_threshold:
            right_fit = np.polyfit(righty, rightx, 2)

        return left_fit, right_fit

    def find_lane_lines(self, binary_warped):
        if (self.left_fit is None) and (self.right_fit is None):
            left_fit, right_fit = self.find_lane_lines_first(binary_warped)
            if left_fit is not None:
                self.left_fit = left_fit
            if right_fit is not None:
                self.right_fit = right_fit
        else:
            #left_fit, right_fit = self.find_lane_lines_after(binary_warped, self.left_fit, self.right_fit)
            left_fit, right_fit = self.find_lane_lines_first(binary_warped)
            if left_fit is not None:
                self.left_fit = left_fit
            if right_fit is not None:
                self.right_fit = right_fit

        h, w = binary_warped.shape[:2]

        ploty = np.linspace(0, h-1, num=h) # to cover same y-range as image


#        left_fitx = self.left_fit[0]*ploty**2 + self.left_fit[1]*ploty + self.left_fit[2]
#        right_fitx = self.right_fit[0]*ploty**2 + self.right_fit[1]*ploty + self.right_fit[2]

        left_fitx = self.left_line.update(self.left_fit, ploty)
        right_fitx = self.right_line.update(self.right_fit, ploty)

        # Create an image to draw the lines on
        warp_zero = np.zeros_like(binary_warped).astype(np.uint8)
        color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
        if left_fitx is not None and right_fitx is not None:

            # Recast the x and y points into usable format for cv2.fillPoly()
            pts_left = np.array([np.transpose(np.vstack([left_fitx, ploty]))])
            pts_right = np.array([np.flipud(np.transpose(np.vstack([right_fitx, ploty])))])
            pts = np.hstack((pts_left, pts_right))

            # Draw the lane onto the warped blank image
            cv2.fillPoly(color_warp, np.int_([pts]), (0,255, 0))

        return color_warp

line_found_imgs = []
for idx, fname in enumerate(img_fnames):
    img = binary_warped_imgs[idx]
    lane = Lane()
    dst = lane.find_lane_lines(img)
    line_found_imgs.append(dst)
    
    out = os.path.join('./output_images/line_found', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(binary_warped_imgs[2:], line_found_imgs[2:], "warped", "line_found")

def inv_warp(img, line_img):
    h, w = img.shape[:2]
    
    line_img = recover(line_img)

    inv_warped = cv2.warpPerspective(line_img, Minv, (w, h), flags=cv2.INTER_LINEAR)
    
    result = cv2.addWeighted(img, 1, inv_warped, 0.3, 0)
    return result


final_imgs = []
for idx, fname in enumerate(img_fnames):
    img = undistorted_imgs[idx]
    line_img = line_found_imgs[idx]
    dst = inv_warp(img, line_img)
    final_imgs.append(dst)
    
    out = os.path.join('./output_images/final', os.path.basename(fname))
    cv2.imwrite(out, dst)
    
show_imgs(undistorted_imgs[2:], final_imgs[2:], "undistorted", "final")

lane = None

def pipeline(img):
    undistorted = cv2.undistort(img, mtx, dist, None, mtx)
    combined = combined_threshold(undistorted)
    binary_warped = unwarp_trim(combined)
    line_img = lane.find_lane_lines(binary_warped)
    result = inv_warp(img, line_img)
    
    return result

Pipeline on Video

# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML

def process_image(image):
    # NOTE: The output you return should be a color image (3 channel) for processing video below
    # TODO: put your pipeline here,
    # you should return the final output (image where lines are drawn on lanes)
    #result = lane_trace(image)
    result = pipeline(image)
    return result

lane = Lane()
white_output = 'result_project_video.mp4'
clip1 = VideoFileClip("project_video.mp4")
white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
%time white_clip.write_videofile(white_output, audio=False)

[MoviePy] >>>> Building video result_project_video.mp4
[MoviePy] Writing video result_project_video.mp4

100%|█████████▉| 1260/1261 [08:27<00:00,  2.43it/s]

[MoviePy] Done.
[MoviePy] >>>> Video ready: result_project_video.mp4 

CPU times: user 7min 58s, sys: 2min 10s, total: 10min 9s
Wall time: 8min 27s

HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))

Pipeline on Optional Videos

Challenge

img_fnames2 = ['test_images/challenge1.png', 'test_images/challenge2.png',
             'test_images/challenge3.png', 'test_images/challenge4.png']

def hls_select_v2(img, thresh=(0, 255)):
    hls = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
    s_channel = hls[:,:,2]
    l_channel = hls[:,:,1]
    kernel = np.ones((11,11),np.uint8)

    binary_output = np.zeros_like(s_channel)
    binary_output[(s_channel > thresh[0]) & (s_channel <= thresh[1]) | (l_channel >= 200)] = 1
    binary_output = cv2.dilate(binary_output,kernel,iterations = 1)
    return binary_output

def combined_threshold_v2(image):
    # Apply each of the thresholding functions
    gradx = abs_sobel_thresh(image, orient='x', sobel_kernel=5, thresh=(20, 100))
    grady = abs_sobel_thresh(image, orient='y', sobel_kernel=5, thresh=(20, 100))
    mag_binary = mag_thresh(image, sobel_kernel=5, mag_thresh=(30, 100))
    dir_binary = dir_threshold(image, sobel_kernel=15, thresh=(0.7, 1.3))

    hls_binary = hls_select_v2(image, thresh=(100, 255))

    combined = np.zeros_like(dir_binary)

    combined[(hls_binary == 1)&(((gradx == 1) & (grady == 1)) | ((mag_binary == 1) & (dir_binary == 1))) ] = 1

    return combined

original_imgs2=[]
processed_imgs = []

for idx, fname in enumerate(img_fnames2):
    lane = Lane()
    img = cv2.imread(fname)
    original_imgs2.append(img)
    
    dst = combined_threshold_v2(img)

    processed_imgs.append(dst)

show_imgs(original_imgs2, processed_imgs, "original", "undistorted")

lane = None

def pipeline(img):
    undistorted = cv2.undistort(img, mtx, dist, None, mtx)
    combined = combined_threshold(undistorted)
    binary_warped = unwarp_trim(combined)
    line_img = line.find_lane_lines(binary_warped)
    result = inv_warp(img, line_img)
    
    return result

def pipeline_v2(img):
    undistorted = cv2.undistort(img, mtx, dist, None, mtx)
    combined = combined_threshold(undistorted)
    binary_warped = unwarp_trim(combined)
    line_img = lane.find_lane_lines(binary_warped)
    result = inv_warp(img, line_img)
    
    return result

def process_image(image):
    result = pipeline(image)
    return result

lane = Lane()
challenge_output = 'result_challenge_video.mp4'
#clip2 = VideoFileClip('challenge_video.mp4')
#challenge_clip = clip2.fl_image(process_image)
#%time challenge_clip.write_videofile(challenge_output, audio=False)
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(challenge_output))